{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO\n",
    "See CSV Tables\n",
    "https://docutils.sourceforge.io/docs/ref/rst/directives.html#csv-table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-17T21:38:38.357808Z",
     "start_time": "2019-12-17T21:38:37.357112Z"
    },
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating documentation for for the following providers\n",
      "MSSentinel, M365D, Kusto, SecurityGraph, Splunk, ResourceGraph, Sumologic, LocalData\n",
      "Skipping the following providers\n",
      "AzureSentinel, MDE, Mordor, LogAnalytics, AzureSecurityCenter, MDATP\n",
      "Queries for Local Data\n",
      "----------------------\n",
      "\n",
      "===============  ================================  ======================================  ============  =======\n",
      "QueryGroup       Query                             Description                             ReqdParams    Table\n",
      "===============  ================================  ======================================  ============  =======\n",
      "Azure            list_all_signins_geo              List all Azure AD logon events                        -\n",
      "Network          list_azure_network_flows_by_host  List Azure Network flows by host name                 -\n",
      "Network          list_azure_network_flows_by_ip    List Azure Network flows by IP address                -\n",
      "SecurityAlert    list_alerts                       Retrieves list of alerts                              -\n",
      "WindowsSecurity  get_process_tree                  Get process tree for a process                        -\n",
      "WindowsSecurity  list_host_events                  List events failures on host                          -\n",
      "WindowsSecurity  list_host_logon_failures          List logon failures on host                           -\n",
      "WindowsSecurity  list_host_logons                  List logons on host                                   -\n",
      "WindowsSecurity  list_host_processes               List processes on host                                -\n",
      "===============  ================================  ======================================  ============  =======\n",
      "\n",
      "\n",
      "Queries for Microsoft 365 Defender\n",
      "----------------------------------\n",
      "\n",
      "============  ==========================  ==================================================================================================================================  ==================================================================  ===================\n",
      "QueryGroup    Query                       Description                                                                                                                         ReqdParams                                                          Table\n",
      "============  ==========================  ==================================================================================================================================  ==================================================================  ===================\n",
      "MDATP         file_path                   Lists all file events from files in a certain path                                                                                  end (datetime), start (datetime), path (str)                        DeviceProcessEvents\n",
      "MDATP         host_alerts                 Lists alerts by for a specified hostname                                                                                            host_name (str), end (datetime), start (datetime)                   DeviceAlertEvents\n",
      "MDATP         host_connections            Lists alerts by for a specified hostname                                                                                            host_name (str), end (datetime), start (datetime)                   DeviceNetworkEvents\n",
      "MDATP         ip_alerts                   Lists alerts associated with a specified remote IP                                                                                  ip_address (str), end (datetime), start (datetime)                  DeviceAlertEvents\n",
      "MDATP         ip_connections              Lists alerts associated with a specified remote IP                                                                                  ip_address (str), end (datetime), start (datetime)                  DeviceNetworkEvents\n",
      "MDATP         list_alerts                 Retrieves list of alerts                                                                                                            start (datetime), end (datetime)                                    DeviceAlertEvents\n",
      "MDATP         list_connections            Retrieves list of network connections for a host                                                                                    start (datetime), end (datetime)                                    DeviceNetworkEvents\n",
      "MDATP         list_filehash               Lists all file events by hash                                                                                                       file_hash (str), end (datetime), start (datetime)                   DeviceProcessEvents\n",
      "MDATP         list_files                  Lists all file events by filename                                                                                                   file_name (str), end (datetime), start (datetime)                   DeviceProcessEvents\n",
      "MDATP         list_host_processes         Lists all process creations for a host                                                                                              host_name (str), end (datetime), start (datetime)                   DeviceProcessEvents\n",
      "MDATP         process_cmd_line            Lists all processes with a command line containing a string                                                                         cmd_line (str), end (datetime), start (datetime)                    DeviceProcessEvents\n",
      "MDATP         process_creations           Lists all processes created by name or hash                                                                                         end (datetime), start (datetime), process_identifier (str)          DeviceProcessEvents\n",
      "MDATP         process_paths               Lists all processes created from a path                                                                                             file_path (str), end (datetime), start (datetime)                   DeviceProcessEvents\n",
      "MDATP         protocol_connections        Lists alerts associated with a specified protocol                                                                                   end (datetime), start (datetime), protocol (str)                    DeviceNetworkEvents\n",
      "MDATP         sha1_alerts                 Lists alerts associated with a specified SHA1 hash                                                                                  file_hash (str), end (datetime), start (datetime)                   DeviceAlertEvents\n",
      "MDATP         url_alerts                  Lists alerts associated with a specified URL                                                                                        url (str), end (datetime), start (datetime)                         DeviceAlertEvents\n",
      "MDATP         url_connections             Lists alerts associated with a specified URL                                                                                        url (str), end (datetime), start (datetime)                         DeviceNetworkEvents\n",
      "MDATP         user_files                  Lists all files created by a user                                                                                                   end (datetime), start (datetime), account_name (str)                -\n",
      "MDATP         user_logons                 Lists all user logons by user                                                                                                       end (datetime), start (datetime), account_name (str)                -\n",
      "MDATP         user_network                Lists all network connections associated with a user                                                                                end (datetime), start (datetime), account_name (str)                -\n",
      "MDATP         user_processes              Lists all processes created by a user                                                                                               end (datetime), start (datetime), account_name (str)                -\n",
      "MDATPHunting  accessibility_persistence   This query looks for persistence or privilege escalation done using Windows Accessibility features.                                 start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  av_sites                    Pivot from downloads detected by Windows Defender Antivirus to other files downloaded from the same sites                           start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  b64_pe                      Finding base64 encoded PE files header seen in the command line parameters                                                          start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  brute_force                 Look for public IP addresses that failed to logon to a computer multiple times, using multiple accounts, and eventually succeeded.  start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  cve_2018_1000006l           Looks for CVE-2018-1000006 exploitation                                                                                             start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  cve_2018_1111               Looks for CVE-2018-1111 exploitation                                                                                                start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  cve_2018_4878               This query checks for specific processes and domain TLD used in the CVE-2018-4878                                                   start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  doc_with_link               Looks for a Word document attachment, from which a link was clicked, and after which there was a browser download.                  start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  dropbox_link                Looks for user content downloads from dropbox that originate from a link/redirect from a 3rd party site.                            start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  email_link                  Look for links opened from mail apps – if a detection occurred right afterwards                                                     start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  email_smartscreen           Look for links opened from outlook.exe, followed by a browser download and then a SmartScreen app warning                           start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  malware_recycle             Finding attackers hiding malware in the recycle bin.                                                                                start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  network_scans               Looking for high volume queries against a given RemoteIP, per ComputerName, RemotePort and Process                                  start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  powershell_downloads        Finds PowerShell execution events that could involve a download.                                                                    start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  service_account_powershell  Service Accounts Performing Remote PowerShell                                                                                       start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  smartscreen_ignored         Query for SmartScreen URL blocks, where the user has decided to run the malware nontheless.                                         start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  smb_discovery               Query for processes that accessed more than 10 IP addresses over port 445 (SMB) - possibly scanning for network shares.             start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  tor                         Looks for Tor client, or for a common Tor plugin called Meek.                                                                       start (datetime), end (datetime)                                    -\n",
      "MDATPHunting  uncommon_powershell         Find which uncommon Powershell Cmdlets were executed on that machine in a certain time period.                                      timestamp (str), host_name (str), end (datetime), start (datetime)  -\n",
      "MDATPHunting  user_enumeration            The query finds attempts to list users or groups using Net commands                                                                 start (datetime), end (datetime)                                    -\n",
      "============  ==========================  ==================================================================================================================================  ==================================================================  ===================\n",
      "\n",
      "\n",
      "Queries for Microsoft Sentinel\n",
      "------------------------------\n",
      "\n",
      "==================  ================================  ===========================================================================================================  ===============================================================================================================  ===========================\n",
      "QueryGroup          Query                             Description                                                                                                  ReqdParams                                                                                                       Table\n",
      "==================  ================================  ===========================================================================================================  ===============================================================================================================  ===========================\n",
      "Azure               get_vmcomputer_for_host           Gets latest VMComputer record for Host                                                                       host_name (str), end (datetime), start (datetime)                                                                VMComputer\n",
      "Azure               get_vmcomputer_for_ip             Gets latest VMComputer record for IPAddress                                                                  ip_address (str), end (datetime), start (datetime)                                                               VMComputer\n",
      "Azure               list_aad_signins_for_account      Lists Azure AD Signins for Account                                                                           end (datetime), start (datetime), account_name (str)                                                             SigninLogs\n",
      "Azure               list_aad_signins_for_ip           Lists Azure AD Signins for an IP Address                                                                     ip_address_list (list), end (datetime), start (datetime)                                                         SigninLogs\n",
      "Azure               list_all_signins_geo              Gets Signin data used by morph charts                                                                        end (datetime), start (datetime)                                                                                 SigninLogs\n",
      "Azure               list_azure_activity_for_account   Lists Azure Activity for Account                                                                             end (datetime), start (datetime), account_name (str)                                                             AzureActivity\n",
      "Azure               list_azure_activity_for_ip        Lists Azure Activity for Caller IP Address(es)                                                               ip_address_list (list), end (datetime), start (datetime)                                                         AzureActivity\n",
      "Azure               list_azure_activity_for_resource  Lists Azure Activity for a Resource                                                                          resource_id (str), end (datetime), start (datetime)                                                              AzureActivity\n",
      "Azure               list_storage_ops_for_hash         no description                                                                                               file_hash (str), end (datetime), start (datetime)                                                                StorageFileLogs\n",
      "Azure               list_storage_ops_for_ip           no description                                                                                               ip_address (str), end (datetime), start (datetime)                                                               StorageFileLogs\n",
      "AzureNetwork        az_net_analytics                  All Azure Network Analytics Data                                                                             end (datetime), start (datetime)                                                                                 AzureNetworkAnalytics_CL\n",
      "AzureNetwork        dns_lookups_for_domain            Dns queries for a domain                                                                                     domain (str), end (datetime), start (datetime)                                                                   DnsEvents\n",
      "AzureNetwork        dns_lookups_for_ip                Dns queries for a domain                                                                                     ip_address (str), end (datetime), start (datetime)                                                               DnsEvents\n",
      "AzureNetwork        dns_lookups_from_ip               Dns queries for a domain                                                                                     ip_address (str), end (datetime), start (datetime)                                                               DnsEvents\n",
      "AzureNetwork        get_heartbeat_for_host            Retrieves latest OMS Heartbeat event for host.                                                               host_name (str)                                                                                                  Heartbeat\n",
      "AzureNetwork        get_heartbeat_for_ip              Retrieves latest OMS Heartbeat event for ip address.                                                         ip_address (str)                                                                                                 Heartbeat\n",
      "AzureNetwork        get_host_for_ip                   Gets the latest AzureNetworkAnalytics interface event for a host.                                            ip_address (str)                                                                                                 AzureNetworkAnalytics_CL\n",
      "AzureNetwork        get_ips_for_host                  Gets the latest AzureNetworkAnalytics interface event for a host.                                            host_name (str)                                                                                                  AzureNetworkAnalytics_CL\n",
      "AzureNetwork        list_azure_network_flows_by_host  Retrieves Azure network analytics flow events.                                                               end (datetime), host_name (str), start (datetime)                                                                AzureNetworkAnalytics_CL\n",
      "AzureNetwork        list_azure_network_flows_by_ip    Retrieves Azure network analytics flow events.                                                               ip_address_list (list), end (datetime), start (datetime)                                                         AzureNetworkAnalytics_CL\n",
      "AzureSentinel       get_bookmark_by_id                Retrieves a single Bookmark by BookmarkId                                                                    bookmark_id (str), end (datetime), start (datetime)                                                              HuntingBookmark\n",
      "AzureSentinel       get_bookmark_by_name              Retrieves one or more Bookmarks by Bookmark Name                                                             bookmark_name (str), end (datetime), start (datetime)                                                            HuntingBookmark\n",
      "AzureSentinel       list_bookmarks                    Retrieves list of bookmarks                                                                                  start (datetime), end (datetime)                                                                                 HuntingBookmark\n",
      "AzureSentinel       list_bookmarks_for_entity         Retrieves bookmarks for entity string                                                                        end (datetime), start (datetime)                                                                                 HuntingBookmark\n",
      "AzureSentinel       list_bookmarks_for_tags           Retrieves Bookmark by one or mare Tags                                                                       end (datetime), start (datetime), bookmark_tags (list)                                                           HuntingBookmark\n",
      "Heartbeat           get_heartbeat_for_host            Retrieves latest OMS Heartbeat event for host.                                                               host_name (str)                                                                                                  Heartbeat\n",
      "Heartbeat           get_heartbeat_for_ip              Retrieves latest OMS Heartbeat event for ip address.                                                         ip_address (str)                                                                                                 Heartbeat\n",
      "Heartbeat           get_info_by_hostname              Deprecated - use 'get_heartbeat_for_host'                                                                    host_name (str), end (datetime), start (datetime)                                                                Heartbeat\n",
      "Heartbeat           get_info_by_ipaddress             Deprecated - use 'get_heartbeat_for_ip'                                                                      ip_address (str), end (datetime), start (datetime)                                                               Heartbeat\n",
      "LinuxAudit          auditd_all                        Extract all audit messages grouped by mssg_id                                                                start (datetime), end (datetime)                                                                                 AuditLog_CL\n",
      "LinuxSyslog         all_syslog                        Returns all syslog activity for a host                                                                       start (datetime), end (datetime)                                                                                 Syslog\n",
      "LinuxSyslog         cron_activity                     All cron activity                                                                                            start (datetime), end (datetime)                                                                                 Syslog\n",
      "LinuxSyslog         list_account_logon_failures       All failed user logon events from an IP address                                                              end (datetime), start (datetime), account_name (str)                                                             Syslog\n",
      "LinuxSyslog         list_host_logon_failures          All failed user logon events on a host                                                                       host_name (str), end (datetime), start (datetime)                                                                Syslog\n",
      "LinuxSyslog         list_ip_logon_failures            All failed user logon events from an IP address                                                              ip_address (str), end (datetime), start (datetime)                                                               Syslog\n",
      "LinuxSyslog         list_logon_failures               All failed user logon events on any host                                                                     start (datetime), end (datetime)                                                                                 Syslog\n",
      "LinuxSyslog         list_logons_for_account           All successful user logon events for account (all hosts)                                                     end (datetime), start (datetime), account_name (str)                                                             Syslog\n",
      "LinuxSyslog         list_logons_for_host              All logon events on a host                                                                                   host_name (str), end (datetime), start (datetime)                                                                Syslog\n",
      "LinuxSyslog         list_logons_for_source_ip         All successful user logon events for source IP (all hosts)                                                   ip_address (str), end (datetime), start (datetime)                                                               Syslog\n",
      "LinuxSyslog         squid_activity                    All squid proxy activity                                                                                     host_name (str), end (datetime), start (datetime)                                                                Syslog\n",
      "LinuxSyslog         sudo_activity                     All sudo activity                                                                                            end (datetime), start (datetime)                                                                                 Syslog\n",
      "LinuxSyslog         user_group_activity               All user/group additions, deletions, and modifications                                                       start (datetime), end (datetime)                                                                                 Syslog\n",
      "LinuxSyslog         user_logon                        All user logon events on a host                                                                              host_name (str), end (datetime), start (datetime)                                                                Syslog\n",
      "MultiDataSource     get_timeseries_anomalies          Time Series filtered anomalies detected using built-in KQL time series function-series_decompose_anomalies   table (str), start (datetime), end (datetime)                                                                    na\n",
      "MultiDataSource     get_timeseries_data               Retrieves TimeSeriesData prepared to use with built-in KQL time series functions                             table (str), start (datetime), end (datetime)                                                                    na\n",
      "MultiDataSource     get_timeseries_decompose          Time Series decomposition and anomalies generated using built-in KQL time series function- series_decompose  table (str), start (datetime), end (datetime)                                                                    na\n",
      "MultiDataSource     plot_timeseries_datawithbaseline  Plot timeseries data using built-in KQL time series decomposition using built-in KQL render method           table (str), start (datetime), end (datetime)                                                                    na\n",
      "MultiDataSource     plot_timeseries_scoreanomolies    Plot timeseries anomaly score using built-in KQL render method                                               table (str), start (datetime), end (datetime)                                                                    na\n",
      "Network             get_heartbeat_for_host            Retrieves latest OMS Heartbeat event for host.                                                               host_name (str)                                                                                                  Heartbeat\n",
      "Network             get_heartbeat_for_ip              Retrieves latest OMS Heartbeat event for ip address.                                                         ip_address (str)                                                                                                 Heartbeat\n",
      "Network             get_host_for_ip                   Gets the latest AzureNetworkAnalytics interface event for a host.                                            ip_address (str)                                                                                                 AzureNetworkAnalytics_CL\n",
      "Network             get_ips_for_host                  Gets the latest AzureNetworkAnalytics interface event for a host.                                            host_name (str)                                                                                                  AzureNetworkAnalytics_CL\n",
      "Network             list_azure_network_flows_by_host  Retrieves Azure network analytics flow events.                                                               end (datetime), host_name (str), start (datetime)                                                                AzureNetworkAnalytics_CL\n",
      "Network             list_azure_network_flows_by_ip    Retrieves Azure network analytics flow events.                                                               ip_address_list (list), end (datetime), start (datetime)                                                         AzureNetworkAnalytics_CL\n",
      "Office365           list_activity_for_account         Lists Office Activity for Account                                                                            end (datetime), start (datetime), account_name (str)                                                             OfficeActivity\n",
      "Office365           list_activity_for_ip              Lists Office Activity for Caller IP Address(es)                                                              ip_address_list (list), end (datetime), start (datetime)                                                         OfficeActivity\n",
      "Office365           list_activity_for_resource        Lists Office Activity for a Resource                                                                         resource_id (str), end (datetime), start (datetime)                                                              OfficeActivity\n",
      "SecurityAlert       get_alert                         Retrieves a single alert by SystemAlertId                                                                    system_alert_id (str)                                                                                            SecurityAlert\n",
      "SecurityAlert       list_alerts                       Retrieves list of alerts                                                                                     end (datetime), start (datetime)                                                                                 SecurityAlert\n",
      "SecurityAlert       list_alerts_counts                Retrieves summary count of alerts by type                                                                    end (datetime), start (datetime)                                                                                 SecurityAlert\n",
      "SecurityAlert       list_alerts_for_ip                Retrieves list of alerts with a common IP Address                                                            source_ip_list (str), end (datetime), start (datetime)                                                           SecurityAlert\n",
      "SecurityAlert       list_related_alerts               Retrieves list of alerts with a common host, account or process                                              end (datetime), start (datetime)                                                                                 SecurityAlert\n",
      "ThreatIntelligence  list_indicators                   Retrieves list of all current indicators.                                                                    start (datetime), end (datetime)                                                                                 ThreatIntelligenceIndicator\n",
      "ThreatIntelligence  list_indicators_by_domain         Retrieves list of indicators by domain                                                                       domain_list (list), end (datetime), start (datetime)                                                             ThreatIntelligenceIndicator\n",
      "ThreatIntelligence  list_indicators_by_email          Retrieves list of indicators by email address                                                                observables (list), end (datetime), start (datetime)                                                             ThreatIntelligenceIndicator\n",
      "ThreatIntelligence  list_indicators_by_filepath       Retrieves list of indicators by file path                                                                    observables (list), end (datetime), start (datetime)                                                             ThreatIntelligenceIndicator\n",
      "ThreatIntelligence  list_indicators_by_hash           Retrieves list of indicators by file hash                                                                    file_hash_list (list), end (datetime), start (datetime)                                                          ThreatIntelligenceIndicator\n",
      "ThreatIntelligence  list_indicators_by_ip             Retrieves list of indicators by IP Address                                                                   ip_address_list (list), end (datetime), start (datetime)                                                         ThreatIntelligenceIndicator\n",
      "ThreatIntelligence  list_indicators_by_url            Retrieves list of indicators by URL                                                                          url_list (list), end (datetime), start (datetime)                                                                ThreatIntelligenceIndicator\n",
      "WindowsSecurity     get_host_logon                    Retrieves the logon event for the session id on the host                                                     logon_session_id (str), host_name (str), end (datetime), start (datetime)                                        SecurityEvent\n",
      "WindowsSecurity     get_parent_process                Retrieves the parent process of a supplied process                                                           logon_session_id (str), process_id (str), host_name (str), process_name (str), end (datetime), start (datetime)  SecurityEvent\n",
      "WindowsSecurity     get_process_tree                  Retrieves the process tree of a supplied process                                                             logon_session_id (str), process_id (str), host_name (str), process_name (str), end (datetime), start (datetime)  SecurityEvent\n",
      "WindowsSecurity     list_all_logons_by_host           account all failed or successful logons to a host                                                            host_name (str), end (datetime), start (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_events                       Retrieves list of all events                                                                                 start (datetime), end (datetime)                                                                                 SecurityEvent\n",
      "WindowsSecurity     list_events_by_id                 Retrieves list of events on a host                                                                           event_list (list), end (datetime), start (datetime)                                                              SecurityEvent\n",
      "WindowsSecurity     list_host_events                  Retrieves list of all events on a host                                                                       host_name (str), end (datetime), start (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_host_events_by_id            Retrieves list of events on a host                                                                           host_name (str), end (datetime), start (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_host_logon_failures          Retrieves the logon failure events on the host                                                               host_name (str), end (datetime), start (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_host_logons                  Retrieves the logon events on the host                                                                       start (datetime), host_name (str), end (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_host_processes               Retrieves list of processes on a host                                                                        host_name (str), end (datetime), start (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_hosts_matching_commandline   Retrieves processes on hosts with matching commandline                                                       commandline (str), process_name (str), end (datetime), start (datetime)                                          SecurityEvent\n",
      "WindowsSecurity     list_logon_attempts_by_account    Retrieves the logon events for an account                                                                    end (datetime), start (datetime), account_name (str)                                                             SecurityEvent\n",
      "WindowsSecurity     list_logon_failures_by_account    Retrieves the logon failure events  for an account                                                           end (datetime), start (datetime), account_name (str)                                                             SecurityEvent\n",
      "WindowsSecurity     list_logons_by_account            Retrieves the logon events for an account                                                                    start (datetime), end (datetime), account_name (str)                                                             SecurityEvent\n",
      "WindowsSecurity     list_matching_processes           Retrieves list of processes matching process name                                                            process_name (str), end (datetime), start (datetime)                                                             SecurityEvent\n",
      "WindowsSecurity     list_other_events                 Retrieves list of events other than logon and process on a host                                              host_name (str), end (datetime), start (datetime)                                                                SecurityEvent\n",
      "WindowsSecurity     list_processes_in_session         Retrieves all processes on the host for a logon session                                                      logon_session_id (str), process_id (str), host_name (str), process_name (str), end (datetime), start (datetime)  SecurityEvent\n",
      "==================  ================================  ===========================================================================================================  ===============================================================================================================  ===========================\n",
      "\n",
      "\n",
      "Queries for Azure Resource Graph\n",
      "--------------------------------\n",
      "\n",
      "=============  ==============================  ====================================================  ===================  =========\n",
      "QueryGroup     Query                           Description                                           ReqdParams           Table\n",
      "=============  ==============================  ====================================================  ===================  =========\n",
      "ResourceGraph  list_detailed_virtual_machines  Retrieves list of VMs with network details                                 resources\n",
      "ResourceGraph  list_public_ips                 Retrieves list of resources with public IP addresses                       resources\n",
      "ResourceGraph  list_resources                  Retrieves list of resources                                                resources\n",
      "ResourceGraph  list_resources_by_api_version   Retrieves list of resources for each API version                           resources\n",
      "ResourceGraph  list_resources_by_type          Retrieves list of resources by type                   resource_type (str)  resources\n",
      "ResourceGraph  list_virtual_machines           Retrieves list of VM resources                                             resources\n",
      "=============  ==============================  ====================================================  ===================  =========\n",
      "\n",
      "\n",
      "Queries for Microsoft Graph\n",
      "---------------------------\n",
      "\n",
      "==================  ====================  ====================================================  ==================================================  =======\n",
      "QueryGroup          Query                 Description                                           ReqdParams                                          Table\n",
      "==================  ====================  ====================================================  ==================================================  =======\n",
      "SecurityGraphAlert  get_alert             Retrieves a single alert by AlertId                   alert_id (str)                                      -\n",
      "SecurityGraphAlert  list_alerts           Retrieves list of alerts                              start (datetime), end (datetime)                    -\n",
      "SecurityGraphAlert  list_alerts_for_file  Retrieves list of alerts for file name, path or hash  end (datetime), start (datetime)                    -\n",
      "SecurityGraphAlert  list_alerts_for_host  Retrieves list of alerts for a hostname or FQDN       host_name (str), end (datetime), start (datetime)   -\n",
      "SecurityGraphAlert  list_alerts_for_ip    Retrieves list of alerts for a IP Address             ip_address (str), end (datetime), start (datetime)  -\n",
      "SecurityGraphAlert  list_alerts_for_user  Retrieves list of alerts for a user account           end (datetime), start (datetime)                    -\n",
      "SecurityGraphAlert  list_related_alerts   Retrieves list of alerts with a common entity         end (datetime), start (datetime)                    -\n",
      "==================  ====================  ====================================================  ==================================================  =======\n",
      "\n",
      "\n",
      "Queries for Splunk\n",
      "------------------\n",
      "\n",
      "==============  =========================  =============================================================  ====================================================  =======\n",
      "QueryGroup      Query                      Description                                                    ReqdParams                                            Table\n",
      "==============  =========================  =============================================================  ====================================================  =======\n",
      "Alerts          list_alerts                Retrieves list of alerts                                       start (datetime), end (datetime)                      -\n",
      "Alerts          list_alerts_for_dest_ip    Retrieves list of alerts with a common destination IP Address  ip_address (str), end (datetime), start (datetime)    -\n",
      "Alerts          list_alerts_for_src_ip     Retrieves list of alerts with a common source IP Address       ip_address (str), end (datetime), start (datetime)    -\n",
      "Alerts          list_alerts_for_user       Retrieves list of alerts with a common username                user (str), end (datetime), start (datetime)          -\n",
      "Alerts          list_all_alerts            Retrieves all configured alerts                                start (datetime), end (datetime)                      -\n",
      "Authentication  list_logon_failures        All failed user logon events on any host                       start (datetime), end (datetime)                      -\n",
      "Authentication  list_logons_for_account    All successful user logon events for account (all hosts)       end (datetime), start (datetime), account_name (str)  -\n",
      "Authentication  list_logons_for_host       All logon events on a host                                     host_name (str), end (datetime), start (datetime)     -\n",
      "Authentication  list_logons_for_source_ip  All successful user logon events for source IP (all hosts)     ip_address (str), end (datetime), start (datetime)    -\n",
      "SplunkGeneral   get_events_parameterized   Generic parameterized query from index/source                  start (datetime), end (datetime)                      -\n",
      "SplunkGeneral   list_all_datatypes         Summary of all events by index and sourcetype                  start (datetime), end (datetime)                      -\n",
      "SplunkGeneral   list_all_savedsearches     Retrieves all saved searches                                   start (datetime), end (datetime)                      -\n",
      "audittrail      list_all_audittrail        Retrieves all audit trail logs                                 start (datetime), end (datetime)                      -\n",
      "==============  =========================  =============================================================  ====================================================  =======\n",
      "\n",
      "\n",
      "Queries for Sumologic\n",
      "---------------------\n",
      "\n",
      "================  ==================  =======================================  ================================  =======\n",
      "QueryGroup        Query               Description                              ReqdParams                        Table\n",
      "================  ==================  =======================================  ================================  =======\n",
      "SumologicGeneral  list_all_datatypes  Summary of all events by sourceCategory  start (datetime), end (datetime)  -\n",
      "================  ==================  =======================================  ================================  =======\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from msticpy.data import QueryProvider\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "providers = {\n",
    "    \"MSSentinel\": \"Microsoft Sentinel\",\n",
    "    \"M365D\": \"Microsoft 365 Defender\",\n",
    "    \"Kusto\": \"Kusto/Azure Data Explorer\",\n",
    "    \"SecurityGraph\": \"Microsoft Graph\",\n",
    "    \"Splunk\": \"Splunk\",\n",
    "    \"ResourceGraph\": \"Azure Resource Graph\",\n",
    "    \"Sumologic\": \"Sumologic\",\n",
    "    \"LocalData\": \"Local Data\",\n",
    "}\n",
    "\n",
    "prov_list = QueryProvider.list_data_environments()\n",
    "\n",
    "print(\"Generating documentation for for the following providers\")\n",
    "print(\", \".join(list(providers)))\n",
    "print(\"Skipping the following providers\")\n",
    "print(\", \".join(list(set(prov_list) - set(providers))))\n",
    "env_q_stores = {prov: QueryProvider(prov) for prov in providers}\n",
    "\n",
    "\n",
    "query_series = []\n",
    "for env, env_queries in env_q_stores.items():\n",
    "    query_names = env_queries.list_queries()\n",
    "    for query_name in query_names:\n",
    "        q_group, q_name = query_name.split(\".\")\n",
    "        query_path = [env, q_group, q_name]\n",
    "        q_obj = getattr(getattr(env_queries, q_group), q_name)\n",
    "        qry = env_queries.query_store.get_query(q_group + \".\" + q_name)\n",
    "        if \"table\" in qry.default_params:\n",
    "            q_table = qry.default_params[\"table\"].get(\"default\", \"na\").split()[0]\n",
    "        elif \"table\" in qry.required_params:\n",
    "            q_table = qry.required_params[\"table\"].get(\"default\", \"na\").split()[0]\n",
    "        else:\n",
    "            q_table = \"-\"\n",
    "        q_dict = {\n",
    "            \"Environment\": env,\n",
    "            \"QueryGroup\": q_group,\n",
    "            \"Query\": q_name,\n",
    "            \"Description\": qry.description,\n",
    "            \"Req-Params\": \", \".join(\n",
    "                [\n",
    "                    f\"{param} ({p_data.get('type')})\"\n",
    "                    for param, p_data in qry.required_params.items()\n",
    "                ]\n",
    "            ),\n",
    "            # \"OtherParams\": \", \".join([f\"{param}\" for param in qry.default_params]),\n",
    "            \"Table\": q_table,\n",
    "        }\n",
    "        query_series.append(pd.Series(q_dict))\n",
    "\n",
    "query_df = pd.DataFrame(query_series)\n",
    "\n",
    "# env_queries.\n",
    "\n",
    "from tabulate import tabulate\n",
    "\n",
    "for name, grp in query_df.groupby(\"Environment\"):\n",
    "    friendly_name = providers.get(name, name)\n",
    "    print(f\"Queries for {friendly_name}\")\n",
    "    print(\"-\" * len(f\"Queries for {friendly_name}\"))\n",
    "    print(f\"\\nData Environment identifier: {name}\")\n",
    "    print()\n",
    "    print(\n",
    "        tabulate(\n",
    "            grp.drop(columns=\"Environment\"),\n",
    "            headers=\"keys\",\n",
    "            showindex=\"never\",\n",
    "            tablefmt=\"rst\",\n",
    "        )\n",
    "    )\n",
    "    print(\"\\n\")\n",
    "\n",
    "file_name = \"source/data_acquisition/DataQueries.rst\"\n",
    "with open(file_name, \"w\") as rst_file:\n",
    "    rst_file.write(\"Data Queries Reference\\n\")\n",
    "    rst_file.write(\"=\" * len(\"Data Queries Reference\") + \"\\n\\n\")\n",
    "\n",
    "    group_dict = {name: grp for name, group in query_df.groupby(\"Environment\")}\n",
    "    for name, friendly_name in env_q_stores.items():\n",
    "        grp = group_dict[name]\n",
    "        rst_file.write(f\"Queries for {friendly_name}\")\n",
    "        rst_file.write(\"-\" * len(f\"Queries for {friendly_name}\"))\n",
    "        rst_file.write(f\"\\nData Environment identifier: {name}\")\n",
    "        tbl_txt = tabulate(\n",
    "            grp.drop(columns=\"Environment\"),\n",
    "            headers=\"keys\",\n",
    "            showindex=\"never\",\n",
    "            tablefmt=\"rst\",\n",
    "        )\n",
    "        rst_file.write(tbl_txt)\n",
    "        rst_file.write(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-17T21:18:02.142425Z",
     "start_time": "2019-12-17T21:18:02.137405Z"
    }
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrameGroupBy' object has no attribute 'asdict'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13684/24742320.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mquery_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Environment\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mF:\\anaconda\\envs\\msticpy\\lib\\site-packages\\pandas\\core\\groupby\\groupby.py\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, attr)\u001b[0m\n\u001b[0;32m    909\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mattr\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    910\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 911\u001b[1;33m         raise AttributeError(\n\u001b[0m\u001b[0;32m    912\u001b[0m             \u001b[1;34mf\"'{type(self).__name__}' object has no attribute '{attr}'\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    913\u001b[0m         )\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'DataFrameGroupBy' object has no attribute 'asdict'"
     ]
    }
   ],
   "source": [
    "query_df.groupby(\"Environment\").asdict()"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "interpreter": {
   "hash": "2bc37074a50de3994d4ebdf9197e864a43c9c15c9793b7f9f3363bcff9457253"
  },
  "kernelspec": {
   "display_name": "Python (condadev)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
